{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Intermediate Neural Network in Keras"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this notebook, we improve our [introductory shallow net](https://github.com/the-deep-learners/TensorFlow-LiveLessons/blob/master/notebooks/shallow_net_in_keras.ipynb) from Lesson 1 by applying the theory we have covered since. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Set seed for reproducibility"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "np.random.seed(42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import keras\n",
    "from keras.datasets import mnist\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.optimizers import SGD"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "(X_train, y_train), (X_test, y_test) = mnist.load_data()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Preprocess data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train = X_train.reshape(60000, 784).astype('float32')\n",
    "X_test = X_test.reshape(10000, 784).astype('float32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train /= 255\n",
    "X_test /= 255"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "n_classes = 10\n",
    "y_train = keras.utils.to_categorical(y_train, n_classes)\n",
    "y_test = keras.utils.to_categorical(y_test, n_classes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Design neural network architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "model.add(Dense(64, activation='relu', input_shape=(784,)))\n",
    "model.add(Dense(64, activation='relu'))\n",
    "model.add(Dense(10, activation='softmax'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_1 (Dense)              (None, 64)                50240     \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 64)                4160      \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 10)                650       \n",
      "=================================================================\n",
      "Total params: 55,050\n",
      "Trainable params: 55,050\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Configure model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.compile(loss='categorical_crossentropy', optimizer=SGD(lr=0.1), metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.4785 - acc: 0.8642 - val_loss: 0.2507 - val_acc: 0.9255\n",
      "Epoch 2/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.2245 - acc: 0.9354 - val_loss: 0.1930 - val_acc: 0.9436\n",
      "Epoch 3/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.1716 - acc: 0.9500 - val_loss: 0.1506 - val_acc: 0.9547\n",
      "Epoch 4/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.1415 - acc: 0.9586 - val_loss: 0.1313 - val_acc: 0.9602\n",
      "Epoch 5/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.1201 - acc: 0.9651 - val_loss: 0.1280 - val_acc: 0.9614\n",
      "Epoch 6/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.1045 - acc: 0.9697 - val_loss: 0.1061 - val_acc: 0.9669\n",
      "Epoch 7/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0927 - acc: 0.9726 - val_loss: 0.0984 - val_acc: 0.9697\n",
      "Epoch 8/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0826 - acc: 0.9759 - val_loss: 0.0926 - val_acc: 0.9719\n",
      "Epoch 9/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0758 - acc: 0.9774 - val_loss: 0.0904 - val_acc: 0.9732\n",
      "Epoch 10/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0683 - acc: 0.9797 - val_loss: 0.0963 - val_acc: 0.9705\n",
      "Epoch 11/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0631 - acc: 0.9810 - val_loss: 0.0856 - val_acc: 0.9752\n",
      "Epoch 12/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0575 - acc: 0.9832 - val_loss: 0.0839 - val_acc: 0.9749\n",
      "Epoch 13/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0523 - acc: 0.9846 - val_loss: 0.0881 - val_acc: 0.9733\n",
      "Epoch 14/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0488 - acc: 0.9859 - val_loss: 0.0828 - val_acc: 0.9759\n",
      "Epoch 15/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0454 - acc: 0.9869 - val_loss: 0.0844 - val_acc: 0.9735\n",
      "Epoch 16/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0420 - acc: 0.9875 - val_loss: 0.0864 - val_acc: 0.9749\n",
      "Epoch 17/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0400 - acc: 0.9886 - val_loss: 0.0848 - val_acc: 0.9746\n",
      "Epoch 18/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0364 - acc: 0.9894 - val_loss: 0.0748 - val_acc: 0.9775\n",
      "Epoch 19/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0336 - acc: 0.9902 - val_loss: 0.0824 - val_acc: 0.9756\n",
      "Epoch 20/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0315 - acc: 0.9911 - val_loss: 0.0802 - val_acc: 0.9772\n",
      "Epoch 21/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0304 - acc: 0.9914 - val_loss: 0.0791 - val_acc: 0.9759\n",
      "Epoch 22/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0274 - acc: 0.9923 - val_loss: 0.0769 - val_acc: 0.9777\n",
      "Epoch 23/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0255 - acc: 0.9930 - val_loss: 0.0776 - val_acc: 0.9781\n",
      "Epoch 24/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0241 - acc: 0.9937 - val_loss: 0.0783 - val_acc: 0.9771\n",
      "Epoch 25/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0226 - acc: 0.9936 - val_loss: 0.0824 - val_acc: 0.9764\n",
      "Epoch 26/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0212 - acc: 0.9945 - val_loss: 0.0812 - val_acc: 0.9774\n",
      "Epoch 27/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0190 - acc: 0.9954 - val_loss: 0.0795 - val_acc: 0.9784\n",
      "Epoch 28/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0177 - acc: 0.9958 - val_loss: 0.0829 - val_acc: 0.9759\n",
      "Epoch 29/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0166 - acc: 0.9962 - val_loss: 0.0808 - val_acc: 0.9779\n",
      "Epoch 30/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0147 - acc: 0.9970 - val_loss: 0.0836 - val_acc: 0.9774\n",
      "Epoch 31/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0143 - acc: 0.9967 - val_loss: 0.0811 - val_acc: 0.9778\n",
      "Epoch 32/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0127 - acc: 0.9976 - val_loss: 0.0823 - val_acc: 0.9786\n",
      "Epoch 33/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0117 - acc: 0.9977 - val_loss: 0.0843 - val_acc: 0.9772\n",
      "Epoch 34/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0112 - acc: 0.9978 - val_loss: 0.0842 - val_acc: 0.9776\n",
      "Epoch 35/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0104 - acc: 0.9981 - val_loss: 0.0907 - val_acc: 0.9756\n",
      "Epoch 36/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0098 - acc: 0.9981 - val_loss: 0.0853 - val_acc: 0.9775\n",
      "Epoch 37/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0090 - acc: 0.9984 - val_loss: 0.0861 - val_acc: 0.9770\n",
      "Epoch 38/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0081 - acc: 0.9989 - val_loss: 0.0872 - val_acc: 0.9764\n",
      "Epoch 39/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0074 - acc: 0.9991 - val_loss: 0.0918 - val_acc: 0.9768\n",
      "Epoch 40/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0069 - acc: 0.9990 - val_loss: 0.0898 - val_acc: 0.9771\n",
      "Epoch 41/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0068 - acc: 0.9990 - val_loss: 0.0882 - val_acc: 0.9765\n",
      "Epoch 42/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0063 - acc: 0.9993 - val_loss: 0.0909 - val_acc: 0.9765\n",
      "Epoch 43/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0057 - acc: 0.9995 - val_loss: 0.0904 - val_acc: 0.9780\n",
      "Epoch 44/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0051 - acc: 0.9996 - val_loss: 0.0905 - val_acc: 0.9776\n",
      "Epoch 45/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0050 - acc: 0.9996 - val_loss: 0.0917 - val_acc: 0.9773\n",
      "Epoch 46/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0045 - acc: 0.9997 - val_loss: 0.0917 - val_acc: 0.9773\n",
      "Epoch 47/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0043 - acc: 0.9997 - val_loss: 0.0912 - val_acc: 0.9777\n",
      "Epoch 48/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0039 - acc: 0.9998 - val_loss: 0.0943 - val_acc: 0.9769\n",
      "Epoch 49/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0037 - acc: 0.9999 - val_loss: 0.0959 - val_acc: 0.9764\n",
      "Epoch 50/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0036 - acc: 0.9999 - val_loss: 0.0939 - val_acc: 0.9780\n",
      "Epoch 51/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0032 - acc: 0.9999 - val_loss: 0.0928 - val_acc: 0.9774\n",
      "Epoch 52/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0032 - acc: 0.9999 - val_loss: 0.0958 - val_acc: 0.9767\n",
      "Epoch 53/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0031 - acc: 0.9999 - val_loss: 0.0953 - val_acc: 0.9779\n",
      "Epoch 54/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0029 - acc: 0.9999 - val_loss: 0.0965 - val_acc: 0.9768\n",
      "Epoch 55/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0027 - acc: 0.9999 - val_loss: 0.0965 - val_acc: 0.9779\n",
      "Epoch 56/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0026 - acc: 0.9999 - val_loss: 0.0954 - val_acc: 0.9776\n",
      "Epoch 57/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0024 - acc: 0.9999 - val_loss: 0.0961 - val_acc: 0.9781\n",
      "Epoch 58/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0024 - acc: 0.9999 - val_loss: 0.0963 - val_acc: 0.9778\n",
      "Epoch 59/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0023 - acc: 1.0000 - val_loss: 0.0983 - val_acc: 0.9775\n",
      "Epoch 60/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0022 - acc: 1.0000 - val_loss: 0.0989 - val_acc: 0.9776\n",
      "Epoch 61/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0021 - acc: 1.0000 - val_loss: 0.0995 - val_acc: 0.9772\n",
      "Epoch 62/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0021 - acc: 1.0000 - val_loss: 0.1005 - val_acc: 0.9770\n",
      "Epoch 63/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0020 - acc: 1.0000 - val_loss: 0.1007 - val_acc: 0.9772\n",
      "Epoch 64/200\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "60000/60000 [==============================] - 1s - loss: 0.0019 - acc: 1.0000 - val_loss: 0.0995 - val_acc: 0.9774\n",
      "Epoch 65/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0019 - acc: 1.0000 - val_loss: 0.1003 - val_acc: 0.9774\n",
      "Epoch 66/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0018 - acc: 1.0000 - val_loss: 0.1015 - val_acc: 0.9777\n",
      "Epoch 67/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0017 - acc: 1.0000 - val_loss: 0.1001 - val_acc: 0.9772\n",
      "Epoch 68/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0017 - acc: 1.0000 - val_loss: 0.1008 - val_acc: 0.9772\n",
      "Epoch 69/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0016 - acc: 1.0000 - val_loss: 0.1028 - val_acc: 0.9772\n",
      "Epoch 70/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0016 - acc: 1.0000 - val_loss: 0.1029 - val_acc: 0.9772\n",
      "Epoch 71/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0016 - acc: 1.0000 - val_loss: 0.1028 - val_acc: 0.9774\n",
      "Epoch 72/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0015 - acc: 1.0000 - val_loss: 0.1032 - val_acc: 0.9773\n",
      "Epoch 73/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0015 - acc: 1.0000 - val_loss: 0.1025 - val_acc: 0.9779\n",
      "Epoch 74/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0015 - acc: 1.0000 - val_loss: 0.1038 - val_acc: 0.9772\n",
      "Epoch 75/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0014 - acc: 1.0000 - val_loss: 0.1037 - val_acc: 0.9773\n",
      "Epoch 76/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0014 - acc: 1.0000 - val_loss: 0.1039 - val_acc: 0.9777\n",
      "Epoch 77/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0014 - acc: 1.0000 - val_loss: 0.1046 - val_acc: 0.9773\n",
      "Epoch 78/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0013 - acc: 1.0000 - val_loss: 0.1044 - val_acc: 0.9773\n",
      "Epoch 79/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0013 - acc: 1.0000 - val_loss: 0.1052 - val_acc: 0.9771\n",
      "Epoch 80/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0013 - acc: 1.0000 - val_loss: 0.1049 - val_acc: 0.9774\n",
      "Epoch 81/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0013 - acc: 1.0000 - val_loss: 0.1077 - val_acc: 0.9770\n",
      "Epoch 82/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0012 - acc: 1.0000 - val_loss: 0.1059 - val_acc: 0.9774\n",
      "Epoch 83/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0012 - acc: 1.0000 - val_loss: 0.1058 - val_acc: 0.9771\n",
      "Epoch 84/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0012 - acc: 1.0000 - val_loss: 0.1064 - val_acc: 0.9773\n",
      "Epoch 85/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0012 - acc: 1.0000 - val_loss: 0.1063 - val_acc: 0.9772\n",
      "Epoch 86/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0012 - acc: 1.0000 - val_loss: 0.1079 - val_acc: 0.9772\n",
      "Epoch 87/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0011 - acc: 1.0000 - val_loss: 0.1066 - val_acc: 0.9774\n",
      "Epoch 88/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0011 - acc: 1.0000 - val_loss: 0.1074 - val_acc: 0.9774\n",
      "Epoch 89/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0011 - acc: 1.0000 - val_loss: 0.1081 - val_acc: 0.9773\n",
      "Epoch 90/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0011 - acc: 1.0000 - val_loss: 0.1079 - val_acc: 0.9775\n",
      "Epoch 91/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0011 - acc: 1.0000 - val_loss: 0.1084 - val_acc: 0.9771\n",
      "Epoch 92/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0010 - acc: 1.0000 - val_loss: 0.1081 - val_acc: 0.9773\n",
      "Epoch 93/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0010 - acc: 1.0000 - val_loss: 0.1087 - val_acc: 0.9773\n",
      "Epoch 94/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0010 - acc: 1.0000 - val_loss: 0.1092 - val_acc: 0.9774\n",
      "Epoch 95/200\n",
      "60000/60000 [==============================] - 1s - loss: 9.9927e-04 - acc: 1.0000 - val_loss: 0.1093 - val_acc: 0.9774\n",
      "Epoch 96/200\n",
      "60000/60000 [==============================] - 1s - loss: 9.8933e-04 - acc: 1.0000 - val_loss: 0.1092 - val_acc: 0.9772\n",
      "Epoch 97/200\n",
      "60000/60000 [==============================] - 1s - loss: 9.7706e-04 - acc: 1.0000 - val_loss: 0.1092 - val_acc: 0.9775\n",
      "Epoch 98/200\n",
      "60000/60000 [==============================] - 1s - loss: 9.6611e-04 - acc: 1.0000 - val_loss: 0.1094 - val_acc: 0.9772\n",
      "Epoch 99/200\n",
      "60000/60000 [==============================] - 1s - loss: 9.4689e-04 - acc: 1.0000 - val_loss: 0.1097 - val_acc: 0.9775\n",
      "Epoch 100/200\n",
      "60000/60000 [==============================] - 1s - loss: 9.3695e-04 - acc: 1.0000 - val_loss: 0.1095 - val_acc: 0.9774\n",
      "Epoch 101/200\n",
      "60000/60000 [==============================] - 1s - loss: 9.3021e-04 - acc: 1.0000 - val_loss: 0.1105 - val_acc: 0.9775\n",
      "Epoch 102/200\n",
      "60000/60000 [==============================] - 1s - loss: 9.1504e-04 - acc: 1.0000 - val_loss: 0.1107 - val_acc: 0.9773\n",
      "Epoch 103/200\n",
      "60000/60000 [==============================] - 1s - loss: 9.1007e-04 - acc: 1.0000 - val_loss: 0.1109 - val_acc: 0.9775\n",
      "Epoch 104/200\n",
      "60000/60000 [==============================] - 1s - loss: 8.9332e-04 - acc: 1.0000 - val_loss: 0.1107 - val_acc: 0.9773\n",
      "Epoch 105/200\n",
      "60000/60000 [==============================] - 1s - loss: 8.8180e-04 - acc: 1.0000 - val_loss: 0.1111 - val_acc: 0.9774\n",
      "Epoch 106/200\n",
      "60000/60000 [==============================] - 1s - loss: 8.7657e-04 - acc: 1.0000 - val_loss: 0.1117 - val_acc: 0.9773\n",
      "Epoch 107/200\n",
      "60000/60000 [==============================] - 1s - loss: 8.6383e-04 - acc: 1.0000 - val_loss: 0.1117 - val_acc: 0.9774\n",
      "Epoch 108/200\n",
      "60000/60000 [==============================] - 1s - loss: 8.5332e-04 - acc: 1.0000 - val_loss: 0.1114 - val_acc: 0.9777\n",
      "Epoch 109/200\n",
      "60000/60000 [==============================] - 1s - loss: 8.4448e-04 - acc: 1.0000 - val_loss: 0.1121 - val_acc: 0.9773\n",
      "Epoch 110/200\n",
      "60000/60000 [==============================] - 1s - loss: 8.3663e-04 - acc: 1.0000 - val_loss: 0.1123 - val_acc: 0.9776\n",
      "Epoch 111/200\n",
      "60000/60000 [==============================] - 1s - loss: 8.2692e-04 - acc: 1.0000 - val_loss: 0.1123 - val_acc: 0.9774\n",
      "Epoch 112/200\n",
      "60000/60000 [==============================] - 1s - loss: 8.1818e-04 - acc: 1.0000 - val_loss: 0.1126 - val_acc: 0.9772\n",
      "Epoch 113/200\n",
      "60000/60000 [==============================] - 1s - loss: 8.1627e-04 - acc: 1.0000 - val_loss: 0.1124 - val_acc: 0.9774\n",
      "Epoch 114/200\n",
      "60000/60000 [==============================] - 1s - loss: 8.0304e-04 - acc: 1.0000 - val_loss: 0.1128 - val_acc: 0.9772\n",
      "Epoch 115/200\n",
      "60000/60000 [==============================] - 1s - loss: 7.9308e-04 - acc: 1.0000 - val_loss: 0.1130 - val_acc: 0.9775\n",
      "Epoch 116/200\n",
      "60000/60000 [==============================] - 1s - loss: 7.8779e-04 - acc: 1.0000 - val_loss: 0.1130 - val_acc: 0.9771\n",
      "Epoch 117/200\n",
      "60000/60000 [==============================] - 1s - loss: 7.8072e-04 - acc: 1.0000 - val_loss: 0.1140 - val_acc: 0.9774\n",
      "Epoch 118/200\n",
      "60000/60000 [==============================] - 1s - loss: 7.6669e-04 - acc: 1.0000 - val_loss: 0.1136 - val_acc: 0.9775\n",
      "Epoch 119/200\n",
      "60000/60000 [==============================] - 1s - loss: 7.6492e-04 - acc: 1.0000 - val_loss: 0.1138 - val_acc: 0.9772\n",
      "Epoch 120/200\n",
      "60000/60000 [==============================] - 1s - loss: 7.5824e-04 - acc: 1.0000 - val_loss: 0.1138 - val_acc: 0.9775\n",
      "Epoch 121/200\n",
      "60000/60000 [==============================] - 1s - loss: 7.5277e-04 - acc: 1.0000 - val_loss: 0.1142 - val_acc: 0.9769\n",
      "Epoch 122/200\n",
      "60000/60000 [==============================] - 1s - loss: 7.4638e-04 - acc: 1.0000 - val_loss: 0.1144 - val_acc: 0.9775\n",
      "Epoch 123/200\n",
      "60000/60000 [==============================] - 1s - loss: 7.3855e-04 - acc: 1.0000 - val_loss: 0.1142 - val_acc: 0.9773\n",
      "Epoch 124/200\n",
      "60000/60000 [==============================] - 1s - loss: 7.3353e-04 - acc: 1.0000 - val_loss: 0.1144 - val_acc: 0.9775\n",
      "Epoch 125/200\n",
      "60000/60000 [==============================] - 1s - loss: 7.2691e-04 - acc: 1.0000 - val_loss: 0.1147 - val_acc: 0.9774\n",
      "Epoch 126/200\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "60000/60000 [==============================] - 1s - loss: 7.1877e-04 - acc: 1.0000 - val_loss: 0.1147 - val_acc: 0.9775\n",
      "Epoch 127/200\n",
      "60000/60000 [==============================] - 1s - loss: 7.1330e-04 - acc: 1.0000 - val_loss: 0.1148 - val_acc: 0.9771\n",
      "Epoch 128/200\n",
      "60000/60000 [==============================] - 1s - loss: 7.0961e-04 - acc: 1.0000 - val_loss: 0.1154 - val_acc: 0.9774\n",
      "Epoch 129/200\n",
      "60000/60000 [==============================] - 1s - loss: 7.0377e-04 - acc: 1.0000 - val_loss: 0.1147 - val_acc: 0.9774\n",
      "Epoch 130/200\n",
      "60000/60000 [==============================] - 1s - loss: 6.9733e-04 - acc: 1.0000 - val_loss: 0.1154 - val_acc: 0.9773\n",
      "Epoch 131/200\n",
      "60000/60000 [==============================] - 1s - loss: 6.9228e-04 - acc: 1.0000 - val_loss: 0.1156 - val_acc: 0.9772\n",
      "Epoch 132/200\n",
      "60000/60000 [==============================] - 1s - loss: 6.8761e-04 - acc: 1.0000 - val_loss: 0.1159 - val_acc: 0.9771\n",
      "Epoch 133/200\n",
      "60000/60000 [==============================] - 1s - loss: 6.8353e-04 - acc: 1.0000 - val_loss: 0.1158 - val_acc: 0.9774\n",
      "Epoch 134/200\n",
      "60000/60000 [==============================] - 1s - loss: 6.7750e-04 - acc: 1.0000 - val_loss: 0.1162 - val_acc: 0.9771\n",
      "Epoch 135/200\n",
      "60000/60000 [==============================] - 1s - loss: 6.7072e-04 - acc: 1.0000 - val_loss: 0.1158 - val_acc: 0.9775\n",
      "Epoch 136/200\n",
      "60000/60000 [==============================] - 1s - loss: 6.6779e-04 - acc: 1.0000 - val_loss: 0.1166 - val_acc: 0.9774\n",
      "Epoch 137/200\n",
      "60000/60000 [==============================] - 1s - loss: 6.6279e-04 - acc: 1.0000 - val_loss: 0.1166 - val_acc: 0.9772\n",
      "Epoch 138/200\n",
      "60000/60000 [==============================] - 1s - loss: 6.5824e-04 - acc: 1.0000 - val_loss: 0.1171 - val_acc: 0.9773\n",
      "Epoch 139/200\n",
      "60000/60000 [==============================] - 1s - loss: 6.5465e-04 - acc: 1.0000 - val_loss: 0.1173 - val_acc: 0.9773\n",
      "Epoch 140/200\n",
      "60000/60000 [==============================] - 1s - loss: 6.5011e-04 - acc: 1.0000 - val_loss: 0.1170 - val_acc: 0.9772\n",
      "Epoch 141/200\n",
      "60000/60000 [==============================] - 1s - loss: 6.4690e-04 - acc: 1.0000 - val_loss: 0.1177 - val_acc: 0.9773\n",
      "Epoch 142/200\n",
      "60000/60000 [==============================] - 1s - loss: 6.4178e-04 - acc: 1.0000 - val_loss: 0.1174 - val_acc: 0.9773\n",
      "Epoch 143/200\n",
      "60000/60000 [==============================] - 1s - loss: 6.3734e-04 - acc: 1.0000 - val_loss: 0.1175 - val_acc: 0.9773\n",
      "Epoch 144/200\n",
      "60000/60000 [==============================] - 1s - loss: 6.3393e-04 - acc: 1.0000 - val_loss: 0.1177 - val_acc: 0.9773\n",
      "Epoch 145/200\n",
      "60000/60000 [==============================] - 1s - loss: 6.2917e-04 - acc: 1.0000 - val_loss: 0.1179 - val_acc: 0.9775\n",
      "Epoch 146/200\n",
      "60000/60000 [==============================] - 1s - loss: 6.2485e-04 - acc: 1.0000 - val_loss: 0.1182 - val_acc: 0.9771\n",
      "Epoch 147/200\n",
      "60000/60000 [==============================] - 1s - loss: 6.2252e-04 - acc: 1.0000 - val_loss: 0.1175 - val_acc: 0.9772\n",
      "Epoch 148/200\n",
      "60000/60000 [==============================] - 1s - loss: 6.1810e-04 - acc: 1.0000 - val_loss: 0.1177 - val_acc: 0.9772\n",
      "Epoch 149/200\n",
      "60000/60000 [==============================] - 1s - loss: 6.1423e-04 - acc: 1.0000 - val_loss: 0.1187 - val_acc: 0.9774\n",
      "Epoch 150/200\n",
      "60000/60000 [==============================] - 1s - loss: 6.0999e-04 - acc: 1.0000 - val_loss: 0.1187 - val_acc: 0.9774\n",
      "Epoch 151/200\n",
      "60000/60000 [==============================] - 1s - loss: 6.0768e-04 - acc: 1.0000 - val_loss: 0.1187 - val_acc: 0.9776\n",
      "Epoch 152/200\n",
      "60000/60000 [==============================] - 1s - loss: 6.0392e-04 - acc: 1.0000 - val_loss: 0.1186 - val_acc: 0.9773\n",
      "Epoch 153/200\n",
      "60000/60000 [==============================] - 1s - loss: 6.0023e-04 - acc: 1.0000 - val_loss: 0.1188 - val_acc: 0.9774\n",
      "Epoch 154/200\n",
      "60000/60000 [==============================] - 1s - loss: 5.9694e-04 - acc: 1.0000 - val_loss: 0.1193 - val_acc: 0.9772\n",
      "Epoch 155/200\n",
      "60000/60000 [==============================] - 1s - loss: 5.9343e-04 - acc: 1.0000 - val_loss: 0.1195 - val_acc: 0.9771\n",
      "Epoch 156/200\n",
      "60000/60000 [==============================] - 2s - loss: 5.9063e-04 - acc: 1.0000 - val_loss: 0.1192 - val_acc: 0.9772\n",
      "Epoch 157/200\n",
      "60000/60000 [==============================] - 2s - loss: 5.8791e-04 - acc: 1.0000 - val_loss: 0.1193 - val_acc: 0.9771\n",
      "Epoch 158/200\n",
      "60000/60000 [==============================] - 2s - loss: 5.8399e-04 - acc: 1.0000 - val_loss: 0.1194 - val_acc: 0.9773\n",
      "Epoch 159/200\n",
      "60000/60000 [==============================] - 2s - loss: 5.8049e-04 - acc: 1.0000 - val_loss: 0.1195 - val_acc: 0.9774\n",
      "Epoch 160/200\n",
      "60000/60000 [==============================] - 1s - loss: 5.7753e-04 - acc: 1.0000 - val_loss: 0.1192 - val_acc: 0.9775\n",
      "Epoch 161/200\n",
      "60000/60000 [==============================] - 1s - loss: 5.7520e-04 - acc: 1.0000 - val_loss: 0.1197 - val_acc: 0.9774\n",
      "Epoch 162/200\n",
      "60000/60000 [==============================] - 1s - loss: 5.7192e-04 - acc: 1.0000 - val_loss: 0.1197 - val_acc: 0.9771\n",
      "Epoch 163/200\n",
      "60000/60000 [==============================] - 1s - loss: 5.6971e-04 - acc: 1.0000 - val_loss: 0.1204 - val_acc: 0.9773\n",
      "Epoch 164/200\n",
      "60000/60000 [==============================] - 1s - loss: 5.6689e-04 - acc: 1.0000 - val_loss: 0.1201 - val_acc: 0.9773\n",
      "Epoch 165/200\n",
      "60000/60000 [==============================] - 1s - loss: 5.6463e-04 - acc: 1.0000 - val_loss: 0.1202 - val_acc: 0.9771\n",
      "Epoch 166/200\n",
      "60000/60000 [==============================] - 1s - loss: 5.6155e-04 - acc: 1.0000 - val_loss: 0.1206 - val_acc: 0.9773\n",
      "Epoch 167/200\n",
      "60000/60000 [==============================] - 1s - loss: 5.5831e-04 - acc: 1.0000 - val_loss: 0.1208 - val_acc: 0.9772\n",
      "Epoch 168/200\n",
      "60000/60000 [==============================] - 1s - loss: 5.5547e-04 - acc: 1.0000 - val_loss: 0.1207 - val_acc: 0.9770\n",
      "Epoch 169/200\n",
      "60000/60000 [==============================] - 1s - loss: 5.5346e-04 - acc: 1.0000 - val_loss: 0.1210 - val_acc: 0.9772\n",
      "Epoch 170/200\n",
      "60000/60000 [==============================] - 1s - loss: 5.5140e-04 - acc: 1.0000 - val_loss: 0.1208 - val_acc: 0.9771\n",
      "Epoch 171/200\n",
      "60000/60000 [==============================] - 1s - loss: 5.4791e-04 - acc: 1.0000 - val_loss: 0.1207 - val_acc: 0.9771\n",
      "Epoch 172/200\n",
      "60000/60000 [==============================] - 1s - loss: 5.4535e-04 - acc: 1.0000 - val_loss: 0.1211 - val_acc: 0.9774\n",
      "Epoch 173/200\n",
      "60000/60000 [==============================] - 1s - loss: 5.4422e-04 - acc: 1.0000 - val_loss: 0.1216 - val_acc: 0.9771\n",
      "Epoch 174/200\n",
      "60000/60000 [==============================] - 1s - loss: 5.4011e-04 - acc: 1.0000 - val_loss: 0.1210 - val_acc: 0.9773\n",
      "Epoch 175/200\n",
      "60000/60000 [==============================] - 1s - loss: 5.3945e-04 - acc: 1.0000 - val_loss: 0.1213 - val_acc: 0.9773\n",
      "Epoch 176/200\n",
      "60000/60000 [==============================] - 1s - loss: 5.3635e-04 - acc: 1.0000 - val_loss: 0.1212 - val_acc: 0.9773\n",
      "Epoch 177/200\n",
      "60000/60000 [==============================] - 1s - loss: 5.3404e-04 - acc: 1.0000 - val_loss: 0.1217 - val_acc: 0.9773\n",
      "Epoch 178/200\n",
      "60000/60000 [==============================] - 1s - loss: 5.3159e-04 - acc: 1.0000 - val_loss: 0.1216 - val_acc: 0.9773\n",
      "Epoch 179/200\n",
      "60000/60000 [==============================] - 1s - loss: 5.2900e-04 - acc: 1.0000 - val_loss: 0.1216 - val_acc: 0.9772\n",
      "Epoch 180/200\n",
      "60000/60000 [==============================] - 1s - loss: 5.2754e-04 - acc: 1.0000 - val_loss: 0.1216 - val_acc: 0.9774\n",
      "Epoch 181/200\n",
      "60000/60000 [==============================] - 1s - loss: 5.2574e-04 - acc: 1.0000 - val_loss: 0.1221 - val_acc: 0.9775\n",
      "Epoch 182/200\n",
      "60000/60000 [==============================] - 1s - loss: 5.2277e-04 - acc: 1.0000 - val_loss: 0.1223 - val_acc: 0.9770\n",
      "Epoch 183/200\n",
      "60000/60000 [==============================] - 1s - loss: 5.2182e-04 - acc: 1.0000 - val_loss: 0.1225 - val_acc: 0.9773\n",
      "Epoch 184/200\n",
      "60000/60000 [==============================] - 1s - loss: 5.1933e-04 - acc: 1.0000 - val_loss: 0.1228 - val_acc: 0.9772\n",
      "Epoch 185/200\n",
      "60000/60000 [==============================] - 1s - loss: 5.1725e-04 - acc: 1.0000 - val_loss: 0.1226 - val_acc: 0.9774\n",
      "Epoch 186/200\n",
      "60000/60000 [==============================] - 1s - loss: 5.1478e-04 - acc: 1.0000 - val_loss: 0.1225 - val_acc: 0.9773\n",
      "Epoch 187/200\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "60000/60000 [==============================] - 1s - loss: 5.1329e-04 - acc: 1.0000 - val_loss: 0.1227 - val_acc: 0.9774\n",
      "Epoch 188/200\n",
      "60000/60000 [==============================] - 1s - loss: 5.1120e-04 - acc: 1.0000 - val_loss: 0.1227 - val_acc: 0.9772\n",
      "Epoch 189/200\n",
      "60000/60000 [==============================] - 1s - loss: 5.0978e-04 - acc: 1.0000 - val_loss: 0.1231 - val_acc: 0.9772\n",
      "Epoch 190/200\n",
      "60000/60000 [==============================] - 1s - loss: 5.0732e-04 - acc: 1.0000 - val_loss: 0.1230 - val_acc: 0.9773\n",
      "Epoch 191/200\n",
      "60000/60000 [==============================] - 1s - loss: 5.0517e-04 - acc: 1.0000 - val_loss: 0.1228 - val_acc: 0.9774\n",
      "Epoch 192/200\n",
      "60000/60000 [==============================] - 1s - loss: 5.0432e-04 - acc: 1.0000 - val_loss: 0.1233 - val_acc: 0.9774\n",
      "Epoch 193/200\n",
      "60000/60000 [==============================] - 1s - loss: 5.0206e-04 - acc: 1.0000 - val_loss: 0.1232 - val_acc: 0.9774\n",
      "Epoch 194/200\n",
      "60000/60000 [==============================] - 1s - loss: 5.0064e-04 - acc: 1.0000 - val_loss: 0.1235 - val_acc: 0.9772\n",
      "Epoch 195/200\n",
      "60000/60000 [==============================] - 1s - loss: 4.9799e-04 - acc: 1.0000 - val_loss: 0.1237 - val_acc: 0.9774\n",
      "Epoch 196/200\n",
      "60000/60000 [==============================] - 1s - loss: 4.9632e-04 - acc: 1.0000 - val_loss: 0.1237 - val_acc: 0.9772\n",
      "Epoch 197/200\n",
      "60000/60000 [==============================] - 1s - loss: 4.9491e-04 - acc: 1.0000 - val_loss: 0.1240 - val_acc: 0.9774\n",
      "Epoch 198/200\n",
      "60000/60000 [==============================] - 1s - loss: 4.9344e-04 - acc: 1.0000 - val_loss: 0.1242 - val_acc: 0.9772\n",
      "Epoch 199/200\n",
      "60000/60000 [==============================] - 1s - loss: 4.9187e-04 - acc: 1.0000 - val_loss: 0.1239 - val_acc: 0.9775\n",
      "Epoch 200/200\n",
      "60000/60000 [==============================] - 1s - loss: 4.8932e-04 - acc: 1.0000 - val_loss: 0.1241 - val_acc: 0.9774\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f3096adadd8>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train, y_train, batch_size=128, epochs=200, verbose=1, validation_data=(X_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
